package com.lcj.crawler;

import com.geccocrawler.gecco.pipeline.Pipeline;
import com.geccocrawler.gecco.request.HttpRequest;
import com.geccocrawler.gecco.scheduler.DeriveSchedulerContext;
import org.springframework.stereotype.Service;

@Service
public class ToCatchPageComp implements Pipeline<Zzz4IndexPage> {
    @Override
    public void process(Zzz4IndexPage bean) {
//        String nLastPageUrl =  bean.getLastPageUrl();    //  /jobs/jobs-list.php?page=50
//        String lastPageUrl = "http://job.zzz4.com"+nLastPageUrl;//http://job.zzz4.com/jobs/jobs-list.php?page=50

       String currentPageUrl = "http://job.hnw.net/jobs/jobs-list.php?page=1";
//       int currentPage = 1;
//       int totalPage = 1;
//       String indexPageUrl = lastPageUrl.split("\\?")[0];       //http://job.hnw.net/jobs/jobs-list.php
//       String totalPageStr = lastPageUrl.split("page=")[1];     //50
//       if(totalPageStr!=null&&!totalPageStr.equals("")){
//           totalPage = Integer.parseInt(totalPageStr);
//       }
//       if(currentPage<=totalPage){
//           currentPageUrl = indexPageUrl+"?page="+currentPage;
//           HttpRequest request = bean.getRequest().subRequest(currentPageUrl);//http://job.hnw.net/jobs/jobs-list.php?page=1
//           DeriveSchedulerContext.into(request);
//           currentPage+=1;
//       }
        HttpRequest request = bean.getRequest().subRequest(currentPageUrl);  //http://job.zzz4.com/jobs/jobs-list.php?page=50
        DeriveSchedulerContext.into(request);
    }
}
